今天的影片內容為解釋向網頁伺服器請求資料失敗可能的原因
以及碰到「反爬蟲機制」的應對方法

以下為影片中有使用到的程式碼

#使用raise_for_status()找出錯誤原因
import requests

url = "https://www.kingstone.com.tw/"
htmlfile = requests.get(url)

if htmlfile.status_code == 200:
    print("列印出網頁內容:\n", htmlfile.text)
else:
    print("網頁下載失敗")
    print(htmlfile.raise_for_status()) 

#新增表頭並偽裝成瀏覽器
import requests

url = "https://www.kingstone.com.tw/"
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36'}
htmlfile = requests.get(url, headers = headers)

if htmlfile.status_code == 200:
    print("列印出網頁內容:\n", htmlfile.text)
else:
    print("網頁下載失敗")
    print("失敗原因:\n", htmlfile.raise_for_status())

#time.sleep
import requests
import time
import random

url = "https://new.ntpu.edu.tw/"
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36'}

for i in range(3):
    
    htmlfile = requests.get(url, headers = headers)
    if htmlfile.status_code == 200:
        print("列印出網頁內容:\n", htmlfile.text)
    else:
        print("網頁下載失敗")
        print("失敗原因:\n", htmlfile.raise_for_status())
    time.sleep(random.randint(1,5))

#設置代理ip
import requests

url = "http://icanhazip.com" #查詢目前ip的網站
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36'}

proxies = {"http":"http://194.5.193.183:80"} #194.5.193.183:80為免費代理ip，不一定有效
htmlfile = requests.get(url, headers = headers, proxies = proxies)

print(htmlfile.text)

#嘗試下載PTT網頁
import requests

url = "https://www.ptt.cc/bbs/Gossiping/index.html"
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36'}
htmlfile = requests.get(url, headers = headers)


if htmlfile.status_code == 200:
    print("列印出網頁內容:\n", htmlfile.text)
else:
    print("網頁下載失敗")
    print("失敗原因:\n", htmlfile.raise_for_status())

#增加cookies
import requests

url = "https://www.ptt.cc/bbs/Gossiping/index.html"
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36'}
cookies = {'over18':'1'}
htmlfile = requests.get(url, headers = headers, cookies = cookies)


if htmlfile.status_code == 200:
    print("列印出網頁內容:\n", htmlfile.text)
else:
    print("網頁下載失敗")
    print("失敗原因:\n", htmlfile.raise_for_status())

推薦代理ip的連結

http://www.google-proxy.net/

如果在影片中有說得不太清楚或錯誤的地方，歡迎留言告訴我，謝謝您的指教。